{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "199b08de",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import json\n",
    "import time\n",
    "import random\n",
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "from urllib.parse import urljoin\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from fake_useragent import UserAgent\n",
    "import logging\n",
    "from logging.handlers import RotatingFileHandler\n",
    "\n",
    "# 配置日志系统\n",
    "def setup_logger():\n",
    "    logger = logging.getLogger('mooc_spider')\n",
    "    logger.setLevel(logging.INFO)\n",
    "    \n",
    "    # 创建文件处理器，限制单个文件大小为5MB，保留3个备份\n",
    "    file_handler = RotatingFileHandler(\n",
    "        'mooc_spider.log', \n",
    "        maxBytes=5*1024*1024, \n",
    "        backupCount=3,\n",
    "        encoding='utf-8'\n",
    "    )\n",
    "    file_handler.setFormatter(logging.Formatter(\n",
    "        '%(asctime)s - %(levelname)s - %(message)s'\n",
    "    ))\n",
    "    \n",
    "    logger.addHandler(file_handler)\n",
    "    return logger\n",
    "\n",
    "logger = setup_logger()\n",
    "\n",
    "class MoocLogin:\n",
    "    \"\"\"处理MOOC登录的类\"\"\"\n",
    "    def __init__(self, username, password):\n",
    "        self.username = username\n",
    "        self.password = password\n",
    "        self.driver = None\n",
    "        self.cookies = None\n",
    "        \n",
    "    def init_driver(self):\n",
    "        \"\"\"初始化Selenium驱动\"\"\"\n",
    "        chrome_options = Options()\n",
    "        chrome_options.add_argument('--headless')\n",
    "        chrome_options.add_argument('--disable-gpu')\n",
    "        chrome_options.add_argument('--no-sandbox')\n",
    "        chrome_options.add_argument('--disable-dev-shm-usage')\n",
    "        \n",
    "        # 随机User-Agent\n",
    "        ua = UserAgent()\n",
    "        chrome_options.add_argument(f'user-agent={ua.random}')\n",
    "        \n",
    "        self.driver = webdriver.Chrome(options=chrome_options)\n",
    "        self.driver.set_page_load_timeout(30)\n",
    "        \n",
    "    def login(self):\n",
    "        \"\"\"执行登录操作\"\"\"\n",
    "        try:\n",
    "            self.init_driver()\n",
    "            login_url = 'https://www.icourse163.org/'\n",
    "            self.driver.get(login_url)\n",
    "            \n",
    "            # 等待登录按钮出现并点击\n",
    "            login_btn = WebDriverWait(self.driver, 10).until(\n",
    "                EC.element_to_be_clickable((By.CSS_SELECTOR, '.ux-login-btn'))\n",
    "            )\n",
    "            login_btn.click()\n",
    "            \n",
    "            # 切换到iframe\n",
    "            WebDriverWait(self.driver, 10).until(\n",
    "                EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, 'iframe'))\n",
    "            )\n",
    "            \n",
    "            # 输入用户名密码\n",
    "            username_input = WebDriverWait(self.driver, 10).until(\n",
    "                EC.presence_of_element_located((By.NAME, 'email'))\n",
    "            )\n",
    "            username_input.send_keys(self.username)\n",
    "            \n",
    "            password_input = self.driver.find_element(By.NAME, 'password')\n",
    "            password_input.send_keys(self.password)\n",
    "            \n",
    "            # 点击登录\n",
    "            submit_btn = self.driver.find_element(By.CSS_SELECTOR, '.j-submit')\n",
    "            submit_btn.click()\n",
    "            \n",
    "            # 等待登录成功\n",
    "            WebDriverWait(self.driver, 10).until(\n",
    "                EC.presence_of_element_located((By.CSS_SELECTOR, '.ux-logined'))\n",
    "            )\n",
    "            \n",
    "            # 获取cookies\n",
    "            self.cookies = {\n",
    "                c['name']: c['value'] \n",
    "                for c in self.driver.get_cookies()\n",
    "            }\n",
    "            \n",
    "            logger.info('登录成功')\n",
    "            return True\n",
    "            \n",
    "        except Exception as e:\n",
    "            logger.error(f'登录失败: {str(e)}')\n",
    "            return False\n",
    "        finally:\n",
    "            if self.driver:\n",
    "                self.driver.quit()\n",
    "                \n",
    "    def get_cookies(self):\n",
    "        \"\"\"获取登录后的cookies\"\"\"\n",
    "        return self.cookies\n",
    "\n",
    "class MoocSpider:\n",
    "    \"\"\"改进后的MOOC爬虫类\"\"\"\n",
    "    def __init__(self, username=None, password=None):\n",
    "        self.session = requests.Session()\n",
    "        self.base_url = 'https://www.icourse163.org/'\n",
    "        self.data_dir = 'mooc_data'\n",
    "        self.proxies = self.load_proxies()\n",
    "        self.current_proxy = None\n",
    "        self.ua = UserAgent()\n",
    "        \n",
    "        # 创建数据目录\n",
    "        os.makedirs(self.data_dir, exist_ok=True)\n",
    "        \n",
    "        # 如果提供了用户名密码，则登录\n",
    "        if username and password:\n",
    "            self.login = MoocLogin(username, password)\n",
    "            if self.login.login():\n",
    "                cookies = self.login.get_cookies()\n",
    "                self.session.cookies.update(cookies)\n",
    "        \n",
    "    def load_proxies(self):\n",
    "        \"\"\"加载代理IP列表\"\"\"\n",
    "        try:\n",
    "            with open('proxies.txt', 'r') as f:\n",
    "                return [line.strip() for line in f if line.strip()]\n",
    "        except FileNotFoundError:\n",
    "            return []\n",
    "    \n",
    "    def rotate_proxy(self):\n",
    "        \"\"\"轮换代理IP\"\"\"\n",
    "        if not self.proxies:\n",
    "            return\n",
    "            \n",
    "        if self.current_proxy:\n",
    "            self.proxies.remove(self.current_proxy)\n",
    "            \n",
    "        if self.proxies:\n",
    "            self.current_proxy = random.choice(self.proxies)\n",
    "            self.session.proxies = {\n",
    "                'http': self.current_proxy,\n",
    "                'https': self.current_proxy\n",
    "            }\n",
    "    \n",
    "    def make_request(self, url, method='get', **kwargs):\n",
    "        \"\"\"封装请求方法，增加重试和代理切换\"\"\"\n",
    "        max_retries = 3\n",
    "        retry_delay = 5\n",
    "        \n",
    "        for attempt in range(max_retries):\n",
    "            try:\n",
    "                # 随机User-Agent\n",
    "                headers = kwargs.get('headers', {})\n",
    "                headers['User-Agent'] = self.ua.random\n",
    "                kwargs['headers'] = headers\n",
    "                \n",
    "                # 随机延迟\n",
    "                time.sleep(random.uniform(1, 3))\n",
    "                \n",
    "                if method.lower() == 'get':\n",
    "                    response = self.session.get(url, **kwargs)\n",
    "                else:\n",
    "                    response = self.session.post(url, **kwargs)\n",
    "                \n",
    "                # 检查响应状态\n",
    "                response.raise_for_status()\n",
    "                \n",
    "                # 检查是否被反爬\n",
    "                if '验证' in response.text or '异常访问' in response.text:\n",
    "                    raise Exception('触发反爬机制')\n",
    "                \n",
    "                return response\n",
    "                \n",
    "            except Exception as e:\n",
    "                logger.warning(f'请求失败 (尝试 {attempt + 1}/{max_retries}): {str(e)}')\n",
    "                if attempt < max_retries - 1:\n",
    "                    # 切换代理\n",
    "                    self.rotate_proxy()\n",
    "                    time.sleep(retry_delay)\n",
    "                else:\n",
    "                    raise\n",
    "        \n",
    "        return None\n",
    "    \n",
    "    def get_course_list(self, keyword, page=1):\n",
    "        \"\"\"获取课程列表\"\"\"\n",
    "        url = f'{self.base_url}web/j/courseBean.getCoursePanelListByFrontCategory.rpc'\n",
    "        params = {\n",
    "            'categoryId': '-1',\n",
    "            'type': '30',\n",
    "            'orderBy': '0',\n",
    "            'pageIndex': page,\n",
    "            'pageSize': '20',\n",
    "            'keyword': keyword\n",
    "        }\n",
    "        \n",
    "        try:\n",
    "            response = self.make_request(url, params=params)\n",
    "            return response.json() if response else None\n",
    "        except Exception as e:\n",
    "            logger.error(f'获取课程列表失败: {str(e)}')\n",
    "            return None\n",
    "    \n",
    "    def get_course_detail(self, course_id):\n",
    "        \"\"\"获取课程详情\"\"\"\n",
    "        url = f'{self.base_url}course/{course_id}'\n",
    "        \n",
    "        try:\n",
    "            response = self.make_request(url)\n",
    "            if not response:\n",
    "                return None\n",
    "                \n",
    "            soup = BeautifulSoup(response.text, 'html.parser')\n",
    "            \n",
    "            # 提取课程基本信息\n",
    "            course_info = {\n",
    "                'id': course_id,\n",
    "                'title': self.clean_text(soup.find('h1').get_text()) if soup.find('h1') else None,\n",
    "                'institution': self.clean_text(soup.find('a', class_='f-fc9').get_text()) if soup.find('a', class_='f-fc9') else None,\n",
    "                'instructor': self.clean_text(soup.find('div', class_='name').get_text()) if soup.find('div', class_='name') else None,\n",
    "                'description': self.clean_text(soup.find('div', class_='course-description').get_text()) if soup.find('div', class_='course-description') else None,\n",
    "                'rating': self.clean_text(soup.find('span', class_='ux-score').get_text()) if soup.find('span', class_='ux-score') else None,\n",
    "                'url': url\n",
    "            }\n",
    "            \n",
    "            # 提取教学大纲\n",
    "            syllabus = []\n",
    "            syllabus_items = soup.select('.j-list .u-chapter')\n",
    "            for item in syllabus_items:\n",
    "                syllabus.append(self.clean_text(item.get_text()))\n",
    "            course_info['syllabus'] = syllabus\n",
    "            \n",
    "            # 提取课程标签\n",
    "            tags = []\n",
    "            tag_items = soup.select('.m-tags .u-tag')\n",
    "            for item in tag_items:\n",
    "                tags.append(self.clean_text(item.get_text()))\n",
    "            course_info['tags'] = tags\n",
    "            \n",
    "            return course_info\n",
    "            \n",
    "        except Exception as e:\n",
    "            logger.error(f'获取课程详情失败 [{course_id}]: {str(e)}')\n",
    "            return None\n",
    "    \n",
    "    def get_course_resources(self, course_id):\n",
    "        \"\"\"获取课程资源(教材、教辅资料等)\"\"\"\n",
    "        url = f'{self.base_url}course/{course_id}/learn#/learn/content'\n",
    "        \n",
    "        try:\n",
    "            response = self.make_request(url)\n",
    "            if not response:\n",
    "                return None\n",
    "                \n",
    "            soup = BeautifulSoup(response.text, 'html.parser')\n",
    "            \n",
    "            resources = []\n",
    "            resource_items = soup.select('.j-list .f-cb')\n",
    "            for item in resource_items:\n",
    "                resource_type = item.select_one('.icon')['class'][1] if item.select_one('.icon') else None\n",
    "                title = self.clean_text(item.select_one('.title').get_text()) if item.select_one('.title') else None\n",
    "                link = urljoin(self.base_url, item.select_one('a')['href']) if item.select_one('a') else None\n",
    "                \n",
    "                resources.append({\n",
    "                    'type': resource_type,\n",
    "                    'title': title,\n",
    "                    'link': link\n",
    "                })\n",
    "            \n",
    "            return resources\n",
    "            \n",
    "        except Exception as e:\n",
    "            logger.error(f'获取课程资源失败 [{course_id}]: {str(e)}')\n",
    "            return None\n",
    "    \n",
    "    def get_exercises(self, course_id):\n",
    "        \"\"\"获取习题库\"\"\"\n",
    "        url = f'{self.base_url}learn/{course_id}/learn#/learn/exercise'\n",
    "        \n",
    "        try:\n",
    "            response = self.make_request(url)\n",
    "            if not response:\n",
    "                return None\n",
    "                \n",
    "            soup = BeautifulSoup(response.text, 'html.parser')\n",
    "            \n",
    "            exercises = []\n",
    "            exercise_items = soup.select('.j-quiz')\n",
    "            for item in exercise_items:\n",
    "                question = self.clean_text(item.select_one('.j-quizTitle').get_text()) if item.select_one('.j-quizTitle') else None\n",
    "                options = [self.clean_text(opt.get_text()) for opt in item.select('.j-option')] if item.select('.j-option') else []\n",
    "                \n",
    "                exercises.append({\n",
    "                    'question': question,\n",
    "                    'options': options\n",
    "                })\n",
    "            \n",
    "            return exercises\n",
    "            \n",
    "        except Exception as e:\n",
    "            logger.error(f'获取习题库失败 [{course_id}]: {str(e)}')\n",
    "            return None\n",
    "    \n",
    "    def clean_text(self, text):\n",
    "        \"\"\"清洗文本\"\"\"\n",
    "        if not text:\n",
    "            return ''\n",
    "            \n",
    "        # 去除HTML标签\n",
    "        text = BeautifulSoup(text, 'html.parser').get_text()\n",
    "        \n",
    "        # 去除特殊字符和多余空格\n",
    "        text = re.sub(r'[\\r\\n\\t]+', ' ', text)\n",
    "        text = re.sub(r'\\s+', ' ', text).strip()\n",
    "        \n",
    "        return text\n",
    "    \n",
    "    def save_data(self, data, filename):\n",
    "        \"\"\"保存数据到文件\"\"\"\n",
    "        filepath = os.path.join(self.data_dir, filename)\n",
    "        \n",
    "        try:\n",
    "            with open(filepath, 'w', encoding='utf-8') as f:\n",
    "                json.dump(data, f, ensure_ascii=False, indent=2)\n",
    "        except Exception as e:\n",
    "            logger.error(f'保存文件失败 [{filename}]: {str(e)}')\n",
    "    \n",
    "    def crawl(self, keyword, max_pages=3):\n",
    "        \"\"\"主爬取方法\"\"\"\n",
    "        all_courses = []\n",
    "        \n",
    "        for page in range(1, max_pages + 1):\n",
    "            logger.info(f'开始爬取第 {page} 页')\n",
    "            \n",
    "            course_list = self.get_course_list(keyword, page)\n",
    "            if not course_list or 'result' not in course_list:\n",
    "                logger.warning(f'第 {page} 页无数据或格式错误')\n",
    "                continue\n",
    "                \n",
    "            for course in course_list['result']['list']:\n",
    "                course_id = course['id']\n",
    "                course_name = course['name']\n",
    "                \n",
    "                logger.info(f'处理课程: {course_name} (ID: {course_id})')\n",
    "                \n",
    "                # 获取课程详情\n",
    "                detail = self.get_course_detail(course_id)\n",
    "                if not detail:\n",
    "                    continue\n",
    "                \n",
    "                # 获取课程资源\n",
    "                resources = self.get_course_resources(course_id)\n",
    "                if resources:\n",
    "                    detail['resources'] = resources\n",
    "                \n",
    "                # 获取习题库\n",
    "                exercises = self.get_exercises(course_id)\n",
    "                if exercises:\n",
    "                    detail['exercises'] = exercises\n",
    "                \n",
    "                all_courses.append(detail)\n",
    "                \n",
    "                # 保存单个课程数据\n",
    "                self.save_data(detail, f'course_{course_id}.json')\n",
    "                \n",
    "                # 随机延迟\n",
    "                time.sleep(random.uniform(2, 5))\n",
    "        \n",
    "        # 保存所有课程数据\n",
    "        self.save_data(all_courses, 'all_courses.json')\n",
    "        logger.info('爬取完成!')\n",
    "        \n",
    "        return all_courses\n",
    "\n",
    "class DataProcessor:\n",
    "    \"\"\"数据预处理类\"\"\"\n",
    "    def __init__(self, data_dir='mooc_data'):\n",
    "        self.data_dir = data_dir\n",
    "        self.stopwords = self.load_stopwords()\n",
    "    \n",
    "    def load_stopwords(self):\n",
    "        \"\"\"加载停用词表\"\"\"\n",
    "        try:\n",
    "            with open('stopwords.txt', 'r', encoding='utf-8') as f:\n",
    "                return set(line.strip() for line in f if line.strip())\n",
    "        except FileNotFoundError:\n",
    "            return set()\n",
    "    \n",
    "    def load_data(self, filename):\n",
    "        \"\"\"加载JSON数据\"\"\"\n",
    "        filepath = os.path.join(self.data_dir, filename)\n",
    "        try:\n",
    "            with open(filepath, 'r', encoding='utf-8') as f:\n",
    "                return json.load(f)\n",
    "        except Exception as e:\n",
    "            logging.error(f'加载数据失败 [{filename}]: {str(e)}')\n",
    "            return None\n",
    "    \n",
    "    def clean_text(self, text):\n",
    "        \"\"\"更完善的文本清洗\"\"\"\n",
    "        if not text:\n",
    "            return ''\n",
    "            \n",
    "        # 去除HTML实体\n",
    "        text = re.sub(r'&[a-z]+;', ' ', text)\n",
    "        \n",
    "        # 去除URL\n",
    "        text = re.sub(r'https?://\\S+|www\\.\\S+', ' ', text)\n",
    "        \n",
    "        # 去除特殊字符\n",
    "        text = re.sub(r'[^\\w\\s\\u4e00-\\u9fff]', ' ', text)\n",
    "        \n",
    "        # 去除多余空格\n",
    "        text = re.sub(r'\\s+', ' ', text).strip()\n",
    "        \n",
    "        return text\n",
    "    \n",
    "    def remove_stopwords(self, text):\n",
    "        \"\"\"去除停用词\"\"\"\n",
    "        if not text:\n",
    "            return ''\n",
    "            \n",
    "        words = text.split()\n",
    "        filtered_words = [word for word in words if word not in self.stopwords]\n",
    "        return ' '.join(filtered_words)\n",
    "    \n",
    "    def process_course_data(self, data):\n",
    "        \"\"\"处理课程数据\"\"\"\n",
    "        processed = []\n",
    "        \n",
    "        for course in data:\n",
    "            try:\n",
    "                # 清洗文本字段\n",
    "                course['title'] = self.clean_text(course.get('title', ''))\n",
    "                course['description'] = self.remove_stopwords(\n",
    "                    self.clean_text(course.get('description', ''))\n",
    "                )\n",
    "                \n",
    "                # 处理教学大纲\n",
    "                if 'syllabus' in course:\n",
    "                    course['syllabus'] = [\n",
    "                        self.clean_text(item) \n",
    "                        for item in course['syllabus']\n",
    "                    ]\n",
    "                \n",
    "                # 处理资源数据\n",
    "                if 'resources' in course:\n",
    "                    for resource in course['resources']:\n",
    "                        resource['title'] = self.clean_text(resource.get('title', ''))\n",
    "                \n",
    "                # 处理习题数据\n",
    "                if 'exercises' in course:\n",
    "                    for exercise in course['exercises']:\n",
    "                        exercise['question'] = self.clean_text(exercise.get('question', ''))\n",
    "                        exercise['options'] = [\n",
    "                            self.clean_text(opt)\n",
    "                            for opt in exercise.get('options', [])\n",
    "                        ]\n",
    "                \n",
    "                processed.append(course)\n",
    "                \n",
    "            except Exception as e:\n",
    "                logging.error(f'处理课程数据失败 [{course.get(\"id\", \"unknown\")}]: {str(e)}')\n",
    "                continue\n",
    "        \n",
    "        return processed\n",
    "    \n",
    "    def extract_key_info(self, data):\n",
    "        \"\"\"提取关键信息到DataFrame\"\"\"\n",
    "        key_info = []\n",
    "        \n",
    "        for course in data:\n",
    "            try:\n",
    "                info = {\n",
    "                    'id': course.get('id'),\n",
    "                    'title': course.get('title'),\n",
    "                    'institution': course.get('institution'),\n",
    "                    'instructor': course.get('instructor'),\n",
    "                    'rating': float(course.get('rating', 0)) if course.get('rating') else 0,\n",
    "                    'resource_count': len(course.get('resources', [])),\n",
    "                    'exercise_count': len(course.get('exercises', [])),\n",
    "                    'chapter_count': len(course.get('syllabus', [])),\n",
    "                    'tag_count': len(course.get('tags', [])),\n",
    "                    'url': course.get('url')\n",
    "                }\n",
    "                key_info.append(info)\n",
    "            except Exception as e:\n",
    "                logging.error(f'提取关键信息失败 [{course.get(\"id\", \"unknown\")}]: {str(e)}')\n",
    "                continue\n",
    "        \n",
    "        return pd.DataFrame(key_info)\n",
    "    \n",
    "    def save_processed_data(self, data, filename):\n",
    "        \"\"\"保存处理后的数据\"\"\"\n",
    "        filepath = os.path.join(self.data_dir, f'processed_{filename}')\n",
    "        try:\n",
    "            with open(filepath, 'w', encoding='utf-8') as f:\n",
    "                json.dump(data, f, ensure_ascii=False, indent=2)\n",
    "        except Exception as e:\n",
    "            logging.error(f'保存处理后的数据失败 [{filename}]: {str(e)}')\n",
    "    \n",
    "    def process_all(self):\n",
    "        \"\"\"执行完整的数据预处理流程\"\"\"\n",
    "        # 加载原始数据\n",
    "        raw_data = self"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d739ac5c",
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'mooc_data\\\\all_courses.json'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[8], line 170\u001b[0m\n\u001b[0;32m    168\u001b[0m \u001b[38;5;66;03m# 使用示例\u001b[39;00m\n\u001b[0;32m    169\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;18m__name__\u001b[39m \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m__main__\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m--> 170\u001b[0m     viewer \u001b[38;5;241m=\u001b[39m \u001b[43mMoocDataViewer\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    172\u001b[0m     \u001b[38;5;66;03m# 显示所有可视化结果\u001b[39;00m\n\u001b[0;32m    173\u001b[0m     viewer\u001b[38;5;241m.\u001b[39mshow_basic_stats()\n",
      "Cell \u001b[1;32mIn[8], line 16\u001b[0m, in \u001b[0;36mMoocDataViewer.__init__\u001b[1;34m(self, data_dir)\u001b[0m\n\u001b[0;32m     14\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;28mself\u001b[39m, data_dir\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mmooc_data\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[0;32m     15\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata_dir \u001b[38;5;241m=\u001b[39m data_dir\n\u001b[1;32m---> 16\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mraw_data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_load_data\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mall_courses.json\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m     17\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mprocessed_data \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_load_data(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mprocessed_all_courses.json\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m     18\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkey_info \u001b[38;5;241m=\u001b[39m pd\u001b[38;5;241m.\u001b[39mread_csv(os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(data_dir, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mkey_info.csv\u001b[39m\u001b[38;5;124m'\u001b[39m))\n",
      "Cell \u001b[1;32mIn[8], line 23\u001b[0m, in \u001b[0;36mMoocDataViewer._load_data\u001b[1;34m(self, filename)\u001b[0m\n\u001b[0;32m     21\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"加载JSON数据\"\"\"\u001b[39;00m\n\u001b[0;32m     22\u001b[0m filepath \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdata_dir, filename)\n\u001b[1;32m---> 23\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28;43mopen\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mfilepath\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mutf-8\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m f:\n\u001b[0;32m     24\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m json\u001b[38;5;241m.\u001b[39mload(f)\n",
      "File \u001b[1;32m~\\AppData\\Roaming\\Python\\Python39\\site-packages\\IPython\\core\\interactiveshell.py:310\u001b[0m, in \u001b[0;36m_modified_open\u001b[1;34m(file, *args, **kwargs)\u001b[0m\n\u001b[0;32m    303\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m file \u001b[38;5;129;01min\u001b[39;00m {\u001b[38;5;241m0\u001b[39m, \u001b[38;5;241m1\u001b[39m, \u001b[38;5;241m2\u001b[39m}:\n\u001b[0;32m    304\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[0;32m    305\u001b[0m         \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mIPython won\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mt let you open fd=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mfile\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m by default \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    306\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mas it is likely to crash IPython. If you know what you are doing, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    307\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124myou can use builtins\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m open.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m    308\u001b[0m     )\n\u001b[1;32m--> 310\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m io_open(file, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'mooc_data\\\\all_courses.json'"
     ]
    }
   ],
   "source": [
    "# mooc_spider.py\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import json\n",
    "import time\n",
    "import random\n",
    "import os\n",
    "import re\n",
    "import pandas as pd\n",
    "from urllib.parse import urljoin\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from fake_useragent import UserAgent\n",
    "import logging\n",
    "from logging.handlers import RotatingFileHandler\n",
    "\n",
    "# 配置日志系统\n",
    "def setup_logger():\n",
    "    logger = logging.getLogger('mooc_spider')\n",
    "    logger.setLevel(logging.INFO)\n",
    "    \n",
    "    file_handler = RotatingFileHandler(\n",
    "        'mooc_spider.log', \n",
    "        maxBytes=5*1024*1024, \n",
    "        backupCount=3,\n",
    "        encoding='utf-8'\n",
    "    )\n",
    "    file_handler.setFormatter(logging.Formatter(\n",
    "        '%(asctime)s - %(levelname)s - %(message)s'\n",
    "    ))\n",
    "    \n",
    "    logger.addHandler(file_handler)\n",
    "    return logger\n",
    "\n",
    "logger = setup_logger()\n",
    "\n",
    "class MoocLogin:\n",
    "    \"\"\"处理MOOC登录的类\"\"\"\n",
    "    def __init__(self, username, password):\n",
    "        self.username = username\n",
    "        self.password = password\n",
    "        self.driver = None\n",
    "        self.cookies = None\n",
    "        \n",
    "    def init_driver(self):\n",
    "        \"\"\"初始化Selenium驱动\"\"\"\n",
    "        chrome_options = Options()\n",
    "        chrome_options.add_argument('--headless')\n",
    "        chrome_options.add_argument('--disable-gpu')\n",
    "        chrome_options.add_argument('--no-sandbox')\n",
    "        chrome_options.add_argument('--disable-dev-shm-usage')\n",
    "        \n",
    "        ua = UserAgent()\n",
    "        chrome_options.add_argument(f'user-agent={ua.random}')\n",
    "        \n",
    "        self.driver = webdriver.Chrome(options=chrome_options)\n",
    "        self.driver.set_page_load_timeout(30)\n",
    "        \n",
    "    def login(self):\n",
    "        \"\"\"执行登录操作\"\"\"\n",
    "        try:\n",
    "            self.init_driver()\n",
    "            login_url = 'https://www.icourse163.org/'\n",
    "            self.driver.get(login_url)\n",
    "            \n",
    "            login_btn = WebDriverWait(self.driver, 10).until(\n",
    "                EC.element_to_be_clickable((By.CSS_SELECTOR, '.ux-login-btn'))\n",
    "            )\n",
    "            login_btn.click()\n",
    "            \n",
    "            WebDriverWait(self.driver, 10).until(\n",
    "                EC.frame_to_be_available_and_switch_to_it((By.CSS_SELECTOR, 'iframe'))\n",
    "            )\n",
    "            \n",
    "            username_input = WebDriverWait(self.driver, 10).until(\n",
    "                EC.presence_of_element_located((By.NAME, 'email'))\n",
    "            )\n",
    "            username_input.send_keys(self.username)\n",
    "            \n",
    "            password_input = self.driver.find_element(By.NAME, 'password')\n",
    "            password_input.send_keys(self.password)\n",
    "            \n",
    "            submit_btn = self.driver.find_element(By.CSS_SELECTOR, '.j-submit')\n",
    "            submit_btn.click()\n",
    "            \n",
    "            WebDriverWait(self.driver, 10).until(\n",
    "                EC.presence_of_element_located((By.CSS_SELECTOR, '.ux-logined'))\n",
    "            )\n",
    "            \n",
    "            self.cookies = {\n",
    "                c['name']: c['value'] \n",
    "                for c in self.driver.get_cookies()\n",
    "            }\n",
    "            \n",
    "            logger.info('登录成功')\n",
    "            return True\n",
    "            \n",
    "        except Exception as e:\n",
    "            logger.error(f'登录失败: {str(e)}')\n",
    "            return False\n",
    "        finally:\n",
    "            if self.driver:\n",
    "                self.driver.quit()\n",
    "                \n",
    "    def get_cookies(self):\n",
    "        return self.cookies\n",
    "\n",
    "class MoocSpider:\n",
    "    \"\"\"MOOC爬虫主类\"\"\"\n",
    "    def __init__(self, username=None, password=None):\n",
    "        self.session = requests.Session()\n",
    "        self.base_url = 'https://www.icourse163.org/'\n",
    "        self.data_dir = 'mooc_data'\n",
    "        self.proxies = self.load_proxies()\n",
    "        self.current_proxy = None\n",
    "        self.ua = UserAgent()\n",
    "        \n",
    "        os.makedirs(self.data_dir, exist_ok=True)\n",
    "        \n",
    "        if username and password:\n",
    "            self.login = MoocLogin(username, password)\n",
    "            if self.login.login():\n",
    "                cookies = self.login.get_cookies()\n",
    "                self.session.cookies.update(cookies)\n",
    "        \n",
    "    def load_proxies(self):\n",
    "        try:\n",
    "            with open('proxies.txt', 'r') as f:\n",
    "                return [line.strip() for line in f if line.strip()]\n",
    "        except FileNotFoundError:\n",
    "            return []\n",
    "    \n",
    "    def rotate_proxy(self):\n",
    "        if not self.proxies:\n",
    "            return\n",
    "            \n",
    "        if self.current_proxy:\n",
    "            self.proxies.remove(self.current_proxy)\n",
    "            \n",
    "        if self.proxies:\n",
    "            self.current_proxy = random.choice(self.proxies)\n",
    "            self.session.proxies = {\n",
    "                'http': self.current_proxy,\n",
    "                'https': self.current_proxy\n",
    "            }\n",
    "    \n",
    "    def make_request(self, url, method='get', **kwargs):\n",
    "        max_retries = 3\n",
    "        retry_delay = 5\n",
    "        \n",
    "        for attempt in range(max_retries):\n",
    "            try:\n",
    "                headers = kwargs.get('headers', {})\n",
    "                headers['User-Agent'] = self.ua.random\n",
    "                kwargs['headers'] = headers\n",
    "                \n",
    "                time.sleep(random.uniform(1, 3))\n",
    "                \n",
    "                if method.lower() == 'get':\n",
    "                    response = self.session.get(url, **kwargs)\n",
    "                else:\n",
    "                    response = self.session.post(url, **kwargs)\n",
    "                \n",
    "                response.raise_for_status()\n",
    "                \n",
    "                if '验证' in response.text or '异常访问' in response.text:\n",
    "                    raise Exception('触发反爬机制')\n",
    "                \n",
    "                return response\n",
    "                \n",
    "            except Exception as e:\n",
    "                logger.warning(f'请求失败 (尝试 {attempt + 1}/{max_retries}): {str(e)}')\n",
    "                if attempt < max_retries - 1:\n",
    "                    self.rotate_proxy()\n",
    "                    time.sleep(retry_delay)\n",
    "                else:\n",
    "                    raise\n",
    "        \n",
    "        return None\n",
    "    \n",
    "    def get_course_list(self, keyword, page=1):\n",
    "        url = f'{self.base_url}web/j/courseBean.getCoursePanelListByFrontCategory.rpc'\n",
    "        params = {\n",
    "            'categoryId': '-1',\n",
    "            'type': '30',\n",
    "            'orderBy': '0',\n",
    "            'pageIndex': page,\n",
    "            'pageSize': '20',\n",
    "            'keyword': keyword\n",
    "        }\n",
    "        \n",
    "        try:\n",
    "            response = self.make_request(url, params=params)\n",
    "            return response.json() if response else None\n",
    "        except Exception as e:\n",
    "            logger.error(f'获取课程列表失败: {str(e)}')\n",
    "            return None\n",
    "    \n",
    "    def get_course_detail(self, course_id):\n",
    "        url = f'{self.base_url}course/{course_id}'\n",
    "        \n",
    "        try:\n",
    "            response = self.make_request(url)\n",
    "            if not response:\n",
    "                return None\n",
    "                \n",
    "            soup = BeautifulSoup(response.text, 'html.parser')\n",
    "            \n",
    "            course_info = {\n",
    "                'id': course_id,\n",
    "                'title': self.clean_text(soup.find('h1').get_text()) if soup.find('h1') else None,\n",
    "                'institution': self.clean_text(soup.find('a', class_='f-fc9').get_text()) if soup.find('a', class_='f-fc9') else None,\n",
    "                'instructor': self.clean_text(soup.find('div', class_='name').get_text()) if soup.find('div', class_='name') else None,\n",
    "                'description': self.clean_text(soup.find('div', class_='course-description').get_text()) if soup.find('div', class_='course-description') else None,\n",
    "                'rating': self.clean_text(soup.find('span', class_='ux-score').get_text()) if soup.find('span', class_='ux-score') else None,\n",
    "                'url': url\n",
    "            }\n",
    "            \n",
    "            syllabus = []\n",
    "            syllabus_items = soup.select('.j-list .u-chapter')\n",
    "            for item in syllabus_items:\n",
    "                syllabus.append(self.clean_text(item.get_text()))\n",
    "            course_info['syllabus'] = syllabus\n",
    "            \n",
    "            tags = []\n",
    "            tag_items = soup.select('.m-tags .u-tag')\n",
    "            for item in tag_items:\n",
    "                tags.append(self.clean_text(item.get_text()))\n",
    "            course_info['tags'] = tags\n",
    "            \n",
    "            return course_info\n",
    "            \n",
    "        except Exception as e:\n",
    "            logger.error(f'获取课程详情失败 [{course_id}]: {str(e)}')\n",
    "            return None\n",
    "    \n",
    "    def get_course_resources(self, course_id):\n",
    "        url = f'{self.base_url}course/{course_id}/learn#/learn/content'\n",
    "        \n",
    "        try:\n",
    "            response = self.make_request(url)\n",
    "            if not response:\n",
    "                return None\n",
    "                \n",
    "            soup = BeautifulSoup(response.text, 'html.parser')\n",
    "            \n",
    "            resources = []\n",
    "            resource_items = soup.select('.j-list .f-cb')\n",
    "            for item in resource_items:\n",
    "                resource_type = item.select_one('.icon')['class'][1] if item.select_one('.icon') else None\n",
    "                title = self.clean_text(item.select_one('.title').get_text()) if item.select_one('.title') else None\n",
    "                link = urljoin(self.base_url, item.select_one('a')['href']) if item.select_one('a') else None\n",
    "                \n",
    "                resources.append({\n",
    "                    'type': resource_type,\n",
    "                    'title': title,\n",
    "                    'link': link\n",
    "                })\n",
    "            \n",
    "            return resources\n",
    "            \n",
    "        except Exception as e:\n",
    "            logger.error(f'获取课程资源失败 [{course_id}]: {str(e)}')\n",
    "            return None\n",
    "    \n",
    "    def get_exercises(self, course_id):\n",
    "        url = f'{self.base_url}learn/{course_id}/learn#/learn/exercise'\n",
    "        \n",
    "        try:\n",
    "            response = self.make_request(url)\n",
    "            if not response:\n",
    "                return None\n",
    "                \n",
    "            soup = BeautifulSoup(response.text, 'html.parser')\n",
    "            \n",
    "            exercises = []\n",
    "            exercise_items = soup.select('.j-quiz')\n",
    "            for item in exercise_items:\n",
    "                question = self.clean_text(item.select_one('.j-quizTitle').get_text()) if item.select_one('.j-quizTitle') else None\n",
    "                options = [self.clean_text(opt.get_text()) for opt in item.select('.j-option')] if item.select('.j-option') else []\n",
    "                \n",
    "                exercises.append({\n",
    "                    'question': question,\n",
    "                    'options': options\n",
    "                })\n",
    "            \n",
    "            return exercises\n",
    "            \n",
    "        except Exception as e:\n",
    "            logger.error(f'获取习题库失败 [{course_id}]: {str(e)}')\n",
    "            return None\n",
    "    \n",
    "    def clean_text(self, text):\n",
    "        if not text:\n",
    "            return ''\n",
    "            \n",
    "        text = BeautifulSoup(text, 'html.parser').get_text()\n",
    "        text = re.sub(r'[\\r\\n\\t]+', ' ', text)\n",
    "        text = re.sub(r'\\s+', ' ', text).strip()\n",
    "        \n",
    "        return text\n",
    "    \n",
    "    def save_data(self, data, filename):\n",
    "        filepath = os.path.join(self.data_dir, filename)\n",
    "        \n",
    "        try:\n",
    "            with open(filepath, 'w', encoding='utf-8') as f:\n",
    "                json.dump(data, f, ensure_ascii=False, indent=2)\n",
    "        except Exception as e:\n",
    "            logger.error(f'保存文件失败 [{filename}]: {str(e)}')\n",
    "    \n",
    "    def crawl(self, keyword, max_pages=3):\n",
    "        all_courses = []\n",
    "        \n",
    "        for page in range(1, max_pages + 1):\n",
    "            logger.info(f'开始爬取第 {page} 页')\n",
    "            \n",
    "            course_list = self.get_course_list(keyword, page)\n",
    "            if not course_list or 'result' not in course_list:\n",
    "                logger.warning(f'第 {page} 页无数据或格式错误')\n",
    "                continue\n",
    "                \n",
    "            for course in course_list['result']['list']:\n",
    "                course_id = course['id']\n",
    "                course_name = course['name']\n",
    "                \n",
    "                logger.info(f'处理课程: {course_name} (ID: {course_id})')\n",
    "                \n",
    "                detail = self.get_course_detail(course_id)\n",
    "                if not detail:\n",
    "                    continue\n",
    "                \n",
    "                resources = self.get_course_resources(course_id)\n",
    "                if resources:\n",
    "                    detail['resources'] = resources\n",
    "                \n",
    "                exercises = self.get_exercises(course_id)\n",
    "                if exercises:\n",
    "                    detail['exercises'] = exercises\n",
    "                \n",
    "                all_courses.append(detail)\n",
    "                \n",
    "                self.save_data(detail, f'course_{course_id}.json')\n",
    "                \n",
    "                time.sleep(random.uniform(2, 5))\n",
    "        \n",
    "        self.save_data(all_courses, 'all_courses.json')\n",
    "        logger.info('爬取完成!')\n",
    "        \n",
    "        return all_courses\n",
    "\n",
    "class DataProcessor:\n",
    "    \"\"\"数据预处理类\"\"\"\n",
    "    def __init__(self, data_dir='mooc_data'):\n",
    "        self.data_dir = data_dir\n",
    "        self.stopwords = self.load_stopwords()\n",
    "    \n",
    "    def load_stopwords(self):\n",
    "        try:\n",
    "            with open('stopwords.txt', 'r', encoding='utf-8') as f:\n",
    "                return set(line.strip() for line in f if line.strip())\n",
    "        except FileNotFoundError:\n",
    "            return set()\n",
    "    \n",
    "    def load_data(self, filename):\n",
    "        filepath = os.path.join(self.data_dir, filename)\n",
    "        try:\n",
    "            with open(filepath, 'r', encoding='utf-8') as f:\n",
    "                return json.load(f)\n",
    "        except Exception as e:\n",
    "            logging.error(f'加载数据失败 [{filename}]: {str(e)}')\n",
    "            return None\n",
    "    \n",
    "    def clean_text(self, text):\n",
    "        if not text:\n",
    "            return ''\n",
    "            \n",
    "        text = re.sub(r'&[a-z]+;', ' ', text)\n",
    "        text = re.sub(r'https?://\\S+|www\\.\\S+', ' ', text)\n",
    "        text = re.sub(r'[^\\w\\s\\u4e00-\\u9fff]', ' ', text)\n",
    "        text = re.sub(r'\\s+', ' ', text).strip()\n",
    "        \n",
    "        return text\n",
    "    \n",
    "    def remove_stopwords(self, text):\n",
    "        if not text:\n",
    "            return ''\n",
    "            \n",
    "        words = text.split()\n",
    "        filtered_words = [word for word in words if word not in self.stopwords]\n",
    "        return ' '.join(filtered_words)\n",
    "    \n",
    "    def process_course_data(self, data):\n",
    "        processed = []\n",
    "        \n",
    "        for course in data:\n",
    "            try:\n",
    "                course['title'] = self.clean_text(course.get('title', ''))\n",
    "                course['description'] = self.remove_stopwords(\n",
    "                    self.clean_text(course.get('description', ''))\n",
    "                )\n",
    "                \n",
    "                if 'syllabus' in course:\n",
    "                    course['syllabus'] = [\n",
    "                        self.clean_text(item) \n",
    "                        for item in course['syllabus']\n",
    "                    ]\n",
    "                \n",
    "                if 'resources' in course:\n",
    "                    for resource in course['resources']:\n",
    "                        resource['title'] = self.clean_text(resource.get('title', ''))\n",
    "                \n",
    "                if 'exercises' in course:\n",
    "                    for exercise in course['exercises']:\n",
    "                        exercise['question'] = self.clean_text(exercise.get('question', ''))\n",
    "                        exercise['options'] = [\n",
    "                            self.clean_text(opt)\n",
    "                            for opt in exercise.get('options', [])\n",
    "                        ]\n",
    "                \n",
    "                processed.append(course)\n",
    "                \n",
    "            except Exception as e:\n",
    "                logging.error(f'处理课程数据失败 [{course.get(\"id\", \"unknown\")}]: {str(e)}')\n",
    "                continue\n",
    "        \n",
    "        return processed\n",
    "    \n",
    "    def extract_key_info(self, data):\n",
    "        key_info = []\n",
    "        \n",
    "        for course in data:\n",
    "            try:\n",
    "                info = {\n",
    "                    'id': course.get('id'),\n",
    "                    'title': course.get('title'),\n",
    "                    'institution': course.get('institution'),\n",
    "                    'instructor': course.get('instructor'),\n",
    "                    'rating': float(course.get('rating', 0)) if course.get('rating') else 0,\n",
    "                    'resource_count': len(course.get('resources', [])),\n",
    "                    'exercise_count': len(course.get('exercises', [])),\n",
    "                    'chapter_count': len(course.get('syllabus', [])),\n",
    "                    'tag_count': len(course.get('tags', [])),\n",
    "                    'url': course.get('url')\n",
    "                }\n",
    "                key_info.append(info)\n",
    "            except Exception as e:\n",
    "                logging.error(f'提取关键信息失败 [{course.get(\"id\", \"unknown\")}]: {str(e)}')\n",
    "                continue\n",
    "        \n",
    "        return pd.DataFrame(key_info)\n",
    "    \n",
    "    def save_processed_data(self, data, filename):\n",
    "        filepath = os.path.join(self.data_dir, f'processed_{filename}')\n",
    "        try:\n",
    "            with open(filepath, 'w', encoding='utf-8') as f:\n",
    "                json.dump(data, f, ensure_ascii=False, indent=2)\n",
    "        except Exception as e:\n",
    "            logging.error(f'保存处理后的数据失败 [{filename}]: {str(e)}')\n",
    "    \n",
    "    def process_all(self):\n",
    "        raw_data = self.load_data('all_courses.json')\n",
    "        if not raw_data:\n",
    "            return None\n",
    "            \n",
    "        cleaned_data = self.process_course_data(raw_data)\n",
    "        self.save_processed_data(cleaned_data, 'cleaned_courses.json')\n",
    "        \n",
    "        key_info_df = self.extract_key_info(cleaned_data)\n",
    "        key_info_df.to_csv(os.path.join(self.data_dir, 'key_info.csv'), index=False)\n",
    "        \n",
    "        print(\"数据预处理完成!\")\n",
    "        return cleaned_data, key_info_df\n",
    "\n",
    "# 使用示例\n",
    "if __name__ == '__main__':\n",
    "    # 1. 爬取数据\n",
    "    spider = MoocSpider(username='你的账号', password='你的密码')\n",
    "    courses = spider.crawl(\"数据挖掘\", max_pages=2)\n",
    "    \n",
    "    # 2. 处理数据\n",
    "    processor = DataProcessor()\n",
    "    cleaned_data, key_info = processor.process_all()\n",
    "    \n",
    "    # 3. 查看结果\n",
    "    print(f\"共处理 {len(cleaned_data)} 门课程\")\n",
    "    print(key_info.head())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
